#ifndef __PAGE_LIB_PRE_PROCESSOR_H__
#define __PAGE_LIB_PRE_PROCESSOR_H__

#include <vector>
#include <unordered_map>
#include <string>
#include "WebPage.h"
#include "cppjieba/Jieba.hpp"
#include "client/create_dict_index/split.h"

using std::vector;
using std::pair;
using std::string;
using std::unordered_map;

class PageLibPreprocessor{
public:
    PageLibPreprocessor();
    void doProcess();
    void readInfoFromFile();
    void cutRedundantPage();
    void buildInvertindexMap();
    void storeOnDisk();

private:
    splitToolCppJieba *_jieba;
    vector<WebPage> _pageList;
    unordered_map<int, pair<int, int>> _offsetLib;
    unordered_map<string, vector<pair<int, double>>> _invertIndexTable;
    unordered_map<string, unordered_map<int, int>> _word_frequnce;//word docId times

public:

};

#endif